package edu.hawaii.webspider;

import java.io.IOException;
import java.net.MalformedURLException;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.Queue;
import java.util.Set;
import java.util.logging.Logger;
import org.xml.sax.SAXException;
import com.meterware.httpunit.GetMethodWebRequest;
import com.meterware.httpunit.WebConversation;
import com.meterware.httpunit.WebLink;
import com.meterware.httpunit.WebRequest;
import com.meterware.httpunit.WebResponse;

/**
 * Iterator for web pages accessible from a starting page.
 * 
 * Does not stop until no additional pages can be reached. You must
 * break out of the for loop when you have enough pages, or this
 * iterator may well download every page on the web.
 * 
 * @author Brian Jaress
 */
public class WebIterator implements Iterable<WebResponse>,
    Iterator<WebResponse> {

  /**
   * Name of the class logger.
   */
  public static final String LOG = "edu.hawaii.webspider.WebIterator";
  private final Logger logger;
  private final WebConversation conversation;
  private final Set<String> pendingOrVisited;
  private final Queue<WebRequest> pending;
  private WebResponse next;

  /**
   * Creates a new WebIterator.
   * 
   */
  public WebIterator() {
    this.pending = new LinkedList<WebRequest>();
    this.pendingOrVisited = new HashSet<String>();
    this.conversation = new WebConversation();
    this.next = null;
    this.logger = Logger.getLogger(LOG);
  }

  /**
   * Sets the start page.
   * 
   * Also clears all history, so pages may be visited again.
   * 
   * @param url the URL of the start page.
   * @throws MalformedURLException on bad URLs.
   * @throws IOException if it can't retrieve the start page.
   * @throws SAXException if it can't parse the start page.
   */
  public void setStartPage(String url) throws MalformedURLException,
      IOException, SAXException {
    this.pending.removeAll(this.pending);
    this.pendingOrVisited.removeAll(this.pendingOrVisited);

    // "prime" the iterator with a first response.
    this.next = this.conversation.getResponse(new GetMethodWebRequest(
        url));
    this.pendingOrVisited.add(url);
    this.logger.info("Got " + url);
  }

  /**
   * Checks whether there are more pages to iterate over.
   * 
   * @return true if there are more pages, false otherwise.
   */
  public boolean hasNext() {
    return !(this.next == null);
  }

  /**
   * Returns the next response in the iteration.
   * 
   * Unlike {@link #setStartPage(URL)}, suppresses retrieval errors.
   * Pages (other than the starting page) that cannot be retrieved are
   * treated as if they were not linked to.
   * 
   * @return the next server response.
   */
  public WebResponse next() {
    WebResponse next = this.next;
    this.makeLinksPending(next);
    this.changeNext();
    return next;
  }

  /**
   * Retrieves a pending page and makes its response the next page in
   * the iteration.
   * 
   * Skips over pages that cannot be retrieved.
   */
  private void changeNext() {
    if (this.next == null) {
      // the iteration is over
      return;
    }
    do {
      try {
        this.next = this.conversation
            .getResponse(this.pending.remove());
        this.logger.info("Got " + this.next.getURL());
      }
      catch (Exception e) {
        this.next = null;
      }
    }
    // keep trying pending pages until we hit one that succeeds or we
    // run out.
    while (this.next == null && !this.pending.isEmpty());
  }

  /**
   * Takes links from the given page and adds them to the queue of
   * pending pages.
   * 
   * Discards links to pages that are already pending or have already
   * been visited. Also keeps track of which pages are pending or
   * visited.
   * 
   * @param response the response containing the page with links.
   */
  private void makeLinksPending(WebResponse response) {
    WebLink[] links;
    try {
      links = response.getLinks();
    }
    catch (Exception e) {
      // there's nothing we can do about a parse error
      return;
    }
    for (WebLink l : links) {
      String url;
      try {
        url = l.getRequest().getURL().toString();
      }
      catch (Exception e) {
        // there's also nothing we can do about malformed URLs
        continue;
      }
      if (!this.pendingOrVisited.contains(url)) {
        this.pending.add(l.getRequest());
        this.pendingOrVisited.add(url);
      }
    }
  }

  /**
   * Not supported.
   * 
   * @throws UnsupportedOperationException always.
   */
  public void remove() {
    throw new UnsupportedOperationException("removal not supported");
  }

  /**
   * Supports for-each looping.
   * 
   * Java for-each loops apparently can't take an Iterator, so we need
   * to support Iterable and return this same object.
   * 
   * @return this.
   */
  public Iterator<WebResponse> iterator() {
    return this;
  }
}
